#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
@Project ：two_layer_crawler 
@File ：settings.py
@IDE  ：PyCharm 
@Author ：李涵彬
@Date ：2024/2/24 16:23 
"""

# 启用Redis调度器和去重器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"

# 配置Redis连接信息
REDIS_URL = 'redis://127.0.0.1:6379'

# 配置MongoDB连接信息
MONGODB_HOST = 'localhost'
MONGODB_PORT = 27017
MONGODB_DB = 'distributed_crawler'
MONGODB_COLLECTION = 'results'

